From c0927948a42621e41da0571209b27ff39badf4cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?B=C5=82a=C5=BCej=20Szczygie=C5=82?= <spaz16@wp.pl>
Date: Sun, 28 Mar 2021 16:19:51 +0200
Subject: [PATCH] VA-API/Vulkan: Treat invalid DRM format modifier as linear

AMD and Mesa 21.0.1 reports invalid DRM modifier instead of linear.
This causes a driver crash when QMPlay2 is trying to play with invalid
image tiling.
---
 src/modules/FFmpeg/VAAPIVulkan.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/modules/FFmpeg/VAAPIVulkan.cpp b/src/modules/FFmpeg/VAAPIVulkan.cpp
index 47ecf7ab..d31dfd66 100644
--- a/src/modules/FFmpeg/VAAPIVulkan.cpp
+++ b/src/modules/FFmpeg/VAAPIVulkan.cpp
@@ -112,8 +112,13 @@ void VAAPIVulkan::map(Frame &frame)
             MemoryObject::FdDescriptors fdDescriptors(vaSurfaceDescr.num_objects);
             for (uint32_t i = 0; i < vaSurfaceDescr.num_objects; ++i)
             {
-                if (i == 0 && vaSurfaceDescr.objects[i].drm_format_modifier != 0)
-                    isLinear = false;
+                if (i == 0)
+                {
+                    // 0x0000000000000000 - linear, 0x00ffffffffffffff - invalid
+                    const auto drmFmtMod = vaSurfaceDescr.objects[i].drm_format_modifier;
+                    if (drmFmtMod != 0ull && drmFmtMod != 0xffffffffffffffull)
+                        isLinear = false;
+                }
 
                 fdDescriptors[i].first = vaSurfaceDescr.objects[i].fd;
                 fdDescriptors[i].second = (vaSurfaceDescr.objects[i].size > 0)
